{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import steward as st\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "import xgboost\n",
    "from sklearn.metrics import roc_auc_score\n",
    "%matplotlib inline\n",
    "from src import build\n",
    "from src import test, config\n",
    "from src.feature_cols import to_drop\n",
    "import h5py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "build.build_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "feature_list = [\n",
    "#     'basic_preprocess/cont_r_test_all',\n",
    "    'basic_preprocess/cont_test_all',\n",
    "    'basic_preprocess/conc_test_all',\n",
    "    \n",
    "    'feature/rank1_test_all',\n",
    "    'feature/rank2_test_all',\n",
    "    'feature/rank3_test_all',\n",
    "    'feature/history1_test_all',\n",
    "    'feature/history2_test_all',\n",
    "    'feature/history3_test_all',\n",
    "\n",
    "    'feature/ordnum_test_all',\n",
    "    \n",
    "    'feature/room1_test_all',\n",
    "    'feature/basicroom1_test_all',\n",
    "    'feature/orderroom1_test_all',\n",
    "    \n",
    "    'feature/rank_history1_test_all',\n",
    "    'feature/rank_history2_test_all',\n",
    "    'feature/rank_history3_test_all',\n",
    "    \n",
    "#     'feature/room1head5_test_all',\n",
    "    'feature/room1small5_test_all',\n",
    "    'feature/room1large5_test_all',\n",
    "#     'feature/orderroom1head5_test_all',\n",
    "    'feature/orderroom1small5_test_all',\n",
    "    'feature/orderroom1large5_test_all',\n",
    "\n",
    "    'feature/custom1_test_all',\n",
    "    'feature/roomcount_date_test_all',\n",
    "\n",
    "\n",
    "]\n",
    "info = 'basic_preprocess/info_test_all'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = h5py.File('data/test.hdf5', 'r')\n",
    "info_df = st.get_instance(info).load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "t = st.LoadInstance('test/real_all_without_head_30W')\n",
    "im = t()['im']\n",
    "select_cols = im.head(395).index.tolist()\n",
    "\n",
    "columns = []\n",
    "for name in feature_list:\n",
    "    columns.extend(f[name].attrs['columns'].astype(str))\n",
    "\n",
    "cols_index = st.tools.get_cols_index(pd.DataFrame(columns=columns), select_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'rank' in select_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 用feature_df\n",
    "cols_index.sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ok!\n"
     ]
    }
   ],
   "source": [
    "test.test_hdf5(f, feature_list, info_df, use_cols_index=cols_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  5.00000000e+02   2.90000000e+01   2.00000000e+02 ...,   0.00000000e+00\n",
      "    2.37777777e+01   1.00000000e+00]\n",
      " [  5.00000000e+02   4.30000000e+01   2.00000000e+02 ...,   0.00000000e+00\n",
      "    3.77777777e+01   1.00000000e+00]\n",
      " [  5.00000000e+02   5.00000000e+00   2.00000000e+02 ...,   0.00000000e+00\n",
      "   -2.22222300e-01   2.00000000e+00]\n",
      " ..., \n",
      " [  5.00000000e+02   1.70000000e+01   2.54000000e+02 ...,   0.00000000e+00\n",
      "    1.08000000e+01   5.00000000e+00]\n",
      " [  4.92000000e+02   3.50000000e+01   2.00000000e+02 ...,   0.00000000e+00\n",
      "    2.88000000e+01   4.00000000e+00]\n",
      " [  5.00000000e+02   4.50000000e+01   2.00000000e+02 ...,   1.00000000e+00\n",
      "    3.88000000e+01   2.00000000e+00]]\n"
     ]
    }
   ],
   "source": [
    "for i, X in enumerate(st.tools.iter_n_rows_list([f[name] for name in feature_list], 100000)):\n",
    "    X = X[:, cols_index]\n",
    "    print(X)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X[:, 1].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for x in f['feature']: print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
